#ifndef __PageLibPreprocessor__
#define __PageLibPreprocessor__

#include<vector>
#include <unordered_map>
#include<iostream>
#include<utility>
#include<map>
#include "PageLib.hh"
#include <regex>
#include "../include/Simhasher.hpp"
#include<stdbool.h>
#include<fstream>
#include "PageLib.hh"
#include <iostream>
#include "WebPage.hh"
#include "tinyxml2.h"
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include "DirScanner.hh"
#include <regex>
#include<unistd.h>
#include "invertedIndex.hh"

using namespace tinyxml2;
using namespace simhash;
using namespace std;

class PageLibPreprocessor
{
public:
    PageLibPreprocessor();
    void doProcess(DirScanner &dirScanner);//预处理
    void readInfoFromFile(DirScanner &dirScanner);//根据配置信息读取网页库和位置偏移库的内容
    uint64_t cutRedundantPages(string &item);//对冗余的网页进行去重
    void buildInvertIndexTable();//创建倒排索引表
    void change();//更新两张表的信息
    void storeOnDisk();//将经过预处理之后的网页库、位置偏移库和倒排索引表写回到磁盘上
    //void insertPage();//加入未初始化网页库
private:
    //TODO
    vector<WebPage> _pageLib;//更新后的网页库
    vector<WebPage> _var;//老的网页库
    map<int,pair<int,int>> _offsetLib;//网页偏移库对象
    unordered_map<string,vector<pair<int,double>>> _inverIndexTable;//倒排索引表对象
    unordered_map<uint64_t,bool> _judge;//判断是否重复
};




#endif